﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

class WebCrawler
{
    private static readonly HttpClient client = new HttpClient();
    private static ConcurrentDictionary<string, bool> visitedUrls = new ConcurrentDictionary<string, bool>();
    private static ConcurrentBag<(string phone, string url)> phoneNumbers = new ConcurrentBag<(string, string)>();
    private static readonly object consoleLock = new object();

    static async Task Main(string[] args)
    {
        Console.WriteLine("高校电话爬虫程序");
        Console.Write("请输入搜索关键字: ");
        string keyword = Console.ReadLine();

        Console.WriteLine("开始爬取...");
        await StartCrawling(keyword);

        Console.WriteLine("\n爬取结果:");
        var groupedResults = phoneNumbers
            .GroupBy(x => x.phone)
            .OrderByDescending(g => g.Count())
            .Take(100);

        foreach (var group in groupedResults)
        {
            Console.WriteLine($"电话号码: {group.Key}");
            Console.WriteLine($"出现次数: {group.Count()}");
            Console.WriteLine("来源URL:");
            foreach (var url in group.Select(x => x.url).Distinct())
            {
                Console.WriteLine($"  - {url}");
            }
            Console.WriteLine();
        }

        Console.WriteLine($"\n已爬取的URL总数: {visitedUrls.Count}");
        Console.WriteLine("按任意键退出...");
        Console.ReadKey();
    }

    static async Task StartCrawling(string keyword)
    {
        string searchUrl = $"https://www.baidu.com/s?wd={Uri.EscapeDataString(keyword + " 高校 电话")}";

        var tasks = new List<Task>();
        tasks.Add(ProcessUrl(searchUrl, 2)); // 从搜索引擎结果开始，设置适当深度

        await Task.WhenAll(tasks);
    }

    static async Task ProcessUrl(string url, int depth)
    {
        if (depth <= 0 || visitedUrls.ContainsKey(url)) return;

        visitedUrls.TryAdd(url, true);
        PrintVisitedUrl(url);

        try
        {
            string html = await client.GetStringAsync(url);
            ExtractPhoneNumbers(html, url);

            if (depth > 1)
            {
                var links = ExtractLinks(html);
                var tasks = links.Select(link => ProcessUrl(link, depth - 1));
                await Task.WhenAll(tasks);
            }
        }
        catch (Exception ex)
        {
            lock (consoleLock)
            {
                Console.ForegroundColor = ConsoleColor.Red;
                Console.WriteLine($"处理URL时出错: {url} - {ex.Message}");
                Console.ResetColor();
            }
        }
    }

    static void ExtractPhoneNumbers(string html, string url)
    {
        // 匹配中国大陆电话号码的正则表达式
        string pattern = @"(?:\(?\d{3,4}\)?[- ]?\d{7,8}|1[3-9]\d{9})";
        var matches = Regex.Matches(html, pattern);

        foreach (Match match in matches)
        {
            string phone = match.Value;
            // 简单验证电话号码有效性
            if (phone.Length >= 7 && phone.Length <= 15)
            {
                phoneNumbers.Add((phone, url));
            }
        }
    }

    static IEnumerable<string> ExtractLinks(string html)
    {
        string pattern = @"href=""(https?://[^""]+)""";
        var matches = Regex.Matches(html, pattern);
        return matches.Select(m => m.Groups[1].Value)
                     .Where(link => link.StartsWith("http") && !visitedUrls.ContainsKey(link))
                     .Distinct();
    }

    static void PrintVisitedUrl(string url)
    {
        lock (consoleLock)
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine($"已访问: {url}");
            Console.ResetColor();
        }
    }
}